package com.finance.cooperate.common.utils;


import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.StringTokenizer;

/**
 * @ClassName SmsPreprocessV1Utils
 * @Description 短信预处理成 Tokens
 * @Author shen
 * @Date 2023/8/2 16:36
 * @Modify ...
 */
public final class SmsPreprocessV1Utils {

    private final static Set<String> stopWords = new HashSet<>();

    /**
     * @Author shen
     * @Description 获取预处理后的短信信息
     * @Date 16:43 2023/8/2
     * @Param [txt]
     * @return java.lang.String
     **/
    public static Set<String> getTokens (String txt) {

        Set<String> result = new HashSet<>();

        txt = preTxt(txt);

        StringTokenizer tokenizer = new StringTokenizer(txt);

        while (tokenizer.hasMoreTokens()) {
            String word = tokenizer.nextToken();
            if (!stopWords.contains(word)) {

                result.add(word);
            }
        }

        return result;
    }

    /**
     * @Author shen
     * @Description 获取预处理后的短信信息
     * @Date 18:10 2023/8/2
     * @Param [txt]
     * @return java.lang.String
     **/
    public static String getTokensJoin (String txt) {

        StringBuilder result = new StringBuilder();

        txt = preTxt(txt);

        StringTokenizer tokenizer = new StringTokenizer(txt);

        while (tokenizer.hasMoreTokens()) {
            String word = tokenizer.nextToken();
            if (!stopWords.contains(word)) {
                result.append(word).append(" ");
            }

        }

        return result.toString().trim();

    }

    /**
     * @Author shen
     * @Description 基本处理
     * 这个处理有问题 地址替换应该放前面，下次更新
     * @Date 18:11 2023/8/2
     * @Param [txt]
     * @return java.lang.String
     **/
    public static String preTxt(String txt) {

        // 去数字替换为x
        txt = txt.replaceAll("[0-9]", " ");
        txt = txt.replaceAll("https?://(?:[-\\w.]|(?:%[\\da-fA-F]{2}))+", " ");
        // 统一小写
        txt = txt.toLowerCase();
        txt = txt.replaceAll("\\n", " ");
        // 去除非英文字符并替换为空格
        txt = txt.replaceAll("[^a-zA-Z]", " ");

        return txt;

    }


    static {

        stopWords.addAll(
                Arrays.asList(
                        "", "no", "x", "hers", "she's", "aren't", "ourselves", "c", "between", "over", "b",
                        "couldn't", "you", "ve", "don", "m", "them", "which", "does", "more", "needn", "it",
                        "mustn", "on", "shan", "such", "g", "because", "each", "nor", "the", "didn't", "up",
                        "but", "from", "and", "hasn't", "or", "you'd", "s", "to", "haven't", "not", "am",
                        "https", "wasn", "those", "xxxxx", "shouldn't", "under", "by", "themselves", "while",
                        "should", "other", "yourself", "didn", "t", "further", "xxx", "u", "down", "won", "xxxxxxxxx",
                        "she", "all", "doesn't", "very", "mustn't", "most", "shouldn", "wouldn't", "that", "again",
                        "he", "re", "will", "z", "don't", "p", "ac", "isn", "were", "h", "y", "this", "through", "an",
                        "me", "any", "k", "being", "than", "at", "i", "him", "q", "their", "d", "against", "v", "now",
                        "its", "been", "my", "these", "ma", "needn't", "above", "xxxx", "before", "com", "mightn't",
                        "own", "as", "has", "wasn't", "just", "her", "how", "we", "f", "into", "once", "did", "o", "of",
                        "is", "myself", "wouldn", "few", "isn't", "where", "be", "yourselves", "after", "who", "couldn",
                        "mightn", "too", "shan't", "e", "same", "his", "it's", "our", "both", "xx", "ain", "weren", "if",
                        "until", "yours", "you've", "your", "have", "hadn't", "only", "theirs", "won't", "off", "some",
                        "having", "you're", "whom", "for", "haven", "a", "with", "aren", "below", "ll", "should've", "hasn",
                        "hadn", "then", "when", "do", "herself", "had", "j", "was", "are", "what", "here", "doesn", "why",
                        "himself", "ours", "can", "weren't", "in", "itself", "that'll", "\n", "doing", "during", "out",
                        "there", "they", "w", "n", "r", "l", "so", "about", "you'll"
                )
        );

    }

}
